{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.datasets\n",
    "import re\n",
    "import numpy as np\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import re\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.util import skipgrams\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "import itertools\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "from scipy.sparse import hstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clearstring(string):\n",
    "    string = re.sub('[^\\'\\\"A-Za-z0-9 ]+', '', string)\n",
    "    string = string.split(' ')\n",
    "    string = filter(None, string)\n",
    "    string = [y.strip() for y in string]\n",
    "    string = ' '.join(string)\n",
    "    return string\n",
    "\n",
    "def separate_dataset(trainset):\n",
    "    datastring = []\n",
    "    datatarget = []\n",
    "    for i in range(len(trainset.data)):\n",
    "        data_ = trainset.data[i].split('\\n')\n",
    "        data_ = list(filter(None, data_))\n",
    "        for n in range(len(data_)):\n",
    "            data_[n] = clearstring(data_[n])\n",
    "        datastring += data_\n",
    "        for n in range(len(data_)):\n",
    "            datatarget.append(trainset.target[i])\n",
    "    return datastring, datatarget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['anger', 'fear', 'joy', 'love', 'sadness', 'surprise']\n",
      "416809\n",
      "416809\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "stopwords= stopwords.words(\"english\")\n",
    "other_exclusions = [\"ff\", \"rt\"]\n",
    "stopwords.extend(other_exclusions)\n",
    "\n",
    "def basic_tokenize(tweet):\n",
    "    \"\"\"Same as tokenize but without the stemming\"\"\"\n",
    "    #return [token.strip() for token in tweet.split()]\n",
    "    #tweet = \" \".join(re.split(\"[^a-zA-Z.,!?]*\", tweet.lower())).strip()\n",
    "    #tweet = \" \".join(re.split(\"[^^a-zA-Z.,!?]*\", tweet)).strip()\n",
    "    tweet = \" \".join(re.split(\"[^a-zA-Z#]+\", tweet)).strip()\n",
    "    #tweet = \" \".join(re.split(\"[ ]*\", tweet)).strip()\n",
    "    return tweet.split()\n",
    "\n",
    "def tokenize(tweet):\n",
    "    tokens = [stemmer.stem(t) for t in tweet.split()]\n",
    "    return tokens\n",
    "\n",
    "def get_metric(vectorizer, X_raw, y_raw, name):\n",
    "    result={'name':name} \n",
    "    y = y_raw\n",
    "    X = vectorizer.fit_transform(X_raw)\n",
    "    result['shape'] = X.shape\n",
    "\n",
    "    aucs = []\n",
    "    for train, test in cv.split(X, y):\n",
    "        classifier.fit(X[train], y[train])\n",
    "        y_preds = classifier.predict(X[test])\n",
    "        accuracy = accuracy_score(y[test], y_preds)\n",
    "        aucs.append(accuracy)\n",
    "\n",
    "    result['accuracies']  = aucs\n",
    "    result['mean_accuracy'] = np.mean(aucs)\n",
    "    #result['y_preds'] = y_preds\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.886839074944823,\n",
       "  0.8853277036752711,\n",
       "  0.8854181661148697,\n",
       "  0.883906722326184,\n",
       "  0.886854921906864,\n",
       "  0.8872867733499676,\n",
       "  0.8856073510712315,\n",
       "  0.8853351888286386,\n",
       "  0.8852632084073132,\n",
       "  0.8841835020874322],\n",
       " 'mean_accuracy': 0.8856022612712593,\n",
       " 'name': 'unigrams-basic',\n",
       " 'shape': (416809, 75300)}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier = LinearSVC(C=1)\n",
    "vectorizer_unigrams = TfidfVectorizer(\n",
    "    ngram_range=(1,1),\n",
    "    stop_words=other_exclusions,\n",
    "    tokenizer=basic_tokenize)\n",
    "\n",
    "result = get_metric(vectorizer_unigrams, np.array(trainset.data), np.array(trainset.target), \"unigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8379234238556761,\n",
       "  0.8410181364552346,\n",
       "  0.836284247396958,\n",
       "  0.8402667818242887,\n",
       "  0.8405508505074254,\n",
       "  0.8389194117223675,\n",
       "  0.8381516758235167,\n",
       "  0.8382599932818273,\n",
       "  0.8379480781227506,\n",
       "  0.83962762128701],\n",
       " 'mean_accuracy': 0.8388950220277055,\n",
       " 'name': 'bigrams-basic',\n",
       " 'shape': (416809, 1078742)}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_bigrams = TfidfVectorizer(\n",
    "    ngram_range=(2,2),\n",
    "    stop_words=other_exclusions,\n",
    "    tokenizer=basic_tokenize)\n",
    "\n",
    "result = get_metric(vectorizer_bigrams, np.array(trainset.data), np.array(trainset.target), \"bigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.717685442855772,\n",
       "  0.7179973131177431,\n",
       "  0.7204308814356317,\n",
       "  0.721342545943093,\n",
       "  0.7177131066912982,\n",
       "  0.7207840502867013,\n",
       "  0.7194405124637124,\n",
       "  0.7181726570372858,\n",
       "  0.7174048658764816,\n",
       "  0.718028696194635],\n",
       " 'mean_accuracy': 0.7189000071902354,\n",
       " 'name': 'trigrams-basic',\n",
       " 'shape': (416809, 3132203)}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_trigrams = TfidfVectorizer(\n",
    "    ngram_range=(3,3),\n",
    "    stop_words=other_exclusions,\n",
    "    tokenizer=basic_tokenize)\n",
    "\n",
    "result = get_metric(vectorizer_trigrams, np.array(trainset.data), np.array(trainset.target), \"trigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def skipgram_tokenize(tweet, n=None, k=None, include_all=True):\n",
    "    tokens = [w for w in basic_tokenize(tweet)]\n",
    "    if include_all:\n",
    "        result = []\n",
    "        for i in range(k+1):\n",
    "            skg = [w for w in skipgrams(tokens, n, i)]\n",
    "            result = result+skg\n",
    "    else:\n",
    "        result = [w for w in skipgrams(tokens, n, k)]\n",
    "    return result\n",
    "\n",
    "def make_skip_tokenize(n, k, include_all=True):\n",
    "    return lambda tweet: skipgram_tokenize(tweet, n=n, k=k, include_all=include_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8546204778812014,\n",
       "  0.857211400057576,\n",
       "  0.8592198071109831,\n",
       "  0.8580682308910321,\n",
       "  0.8574890237758211,\n",
       "  0.8574890237758211,\n",
       "  0.859672272738178,\n",
       "  0.8597581457843466,\n",
       "  0.8564470464033783,\n",
       "  0.858198569988963],\n",
       " 'mean_accuracy': 0.8578173998407301,\n",
       " 'name': '1-skip-bigrams-basic',\n",
       " 'shape': (416809, 2173548)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_1skipbigram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       tokenizer=make_skip_tokenize(n=2, k=1))\n",
    "\n",
    "result = get_metric(vectorizer_1skipbigram, np.array(trainset.data), np.array(trainset.target), \"1-skip-bigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8657278572114001,\n",
       "  0.8690384799923232,\n",
       "  0.8625065975720935,\n",
       "  0.8649297058682405,\n",
       "  0.865694201194789,\n",
       "  0.8641347376502483,\n",
       "  0.8639667954223748,\n",
       "  0.8641009645376457,\n",
       "  0.8640289841163203,\n",
       "  0.865396612121503],\n",
       " 'mean_accuracy': 0.8649524935686937,\n",
       " 'name': '2-skip-bigrams-basic',\n",
       " 'shape': (416809, 3148371)}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_2skipbigram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       tokenizer=make_skip_tokenize(n=2, k=2))\n",
    "    \n",
    "result = get_metric(vectorizer_2skipbigram, np.array(trainset.data), np.array(trainset.target), \"2-skip-bigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8679589290855004,\n",
       "  0.866735438057768,\n",
       "  0.8686723285830814,\n",
       "  0.8700638165155223,\n",
       "  0.8667498380557088,\n",
       "  0.87118831122094,\n",
       "  0.8691490127396175,\n",
       "  0.8675320312874898,\n",
       "  0.8699793656125534,\n",
       "  0.8694275157157253],\n",
       " 'mean_accuracy': 0.8687456586873907,\n",
       " 'name': '3-skip-bigrams-basic',\n",
       " 'shape': (416809, 3970158)}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_3skipbigram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       tokenizer=make_skip_tokenize(n=2, k=3))\n",
    "result = get_metric(vectorizer_3skipbigram, np.array(trainset.data), np.array(trainset.target), \"3-skip-bigrams-basic\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.5996305536896651,\n",
       "  0.5999424239516361,\n",
       "  0.5961326231946643,\n",
       "  0.5985317403195624,\n",
       "  0.599553753508793,\n",
       "  0.5976823972553441,\n",
       "  0.5958350327487344,\n",
       "  0.5985891837420222,\n",
       "  0.5962618167858342,\n",
       "  0.5992849944815011],\n",
       " 'mean_accuracy': 0.5981444519677757,\n",
       " 'name': 'character bigrams',\n",
       " 'shape': (416809, 719)}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_bigram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(2,2))\n",
    "result = get_metric(vectorizer_character_bigram, np.array(trainset.data), np.array(trainset.target), \"character bigrams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8545485078207465,\n",
       "  0.8570674599366663,\n",
       "  0.8541336788061993,\n",
       "  0.8529101290725013,\n",
       "  0.8554017418008205,\n",
       "  0.8570331805858784,\n",
       "  0.8540821957246707,\n",
       "  0.8535678295503623,\n",
       "  0.8528240318633332,\n",
       "  0.8530639666010845],\n",
       " 'mean_accuracy': 0.8544632721762262,\n",
       " 'name': 'character trigrams',\n",
       " 'shape': (416809, 11654)}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_trigram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(3,3))\n",
    "result = get_metric(vectorizer_character_trigram, np.array(trainset.data), np.array(trainset.target), \"character trigrams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8830246617407158,\n",
       "  0.8833605220228385,\n",
       "  0.8822753226812533,\n",
       "  0.8817954992562737,\n",
       "  0.8831841846404836,\n",
       "  0.8833761186151964,\n",
       "  0.8803771502603105,\n",
       "  0.8816641873410432,\n",
       "  0.8830078218724507,\n",
       "  0.8822400307116465],\n",
       " 'mean_accuracy': 0.8824305499142213,\n",
       " 'name': 'character 4-grams',\n",
       " 'shape': (416809, 77730)}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_4gram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(4,4))\n",
    "result = get_metric(vectorizer_character_4gram, np.array(trainset.data), np.array(trainset.target), \"character 4-grams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8752039151712887,\n",
       "  0.8788983782746378,\n",
       "  0.8777889736576939,\n",
       "  0.8748860419365674,\n",
       "  0.8767783882344473,\n",
       "  0.8762505698039874,\n",
       "  0.876418512031861,\n",
       "  0.8748020538413551,\n",
       "  0.8786650031191516,\n",
       "  0.8798406833341331],\n",
       " 'mean_accuracy': 0.8769532519405123,\n",
       " 'name': 'character 5-grams',\n",
       " 'shape': (416809, 282187)}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_5gram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(5,5))\n",
    "result = get_metric(vectorizer_character_5gram, np.array(trainset.data), np.array(trainset.target), \"character 5-grams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8755157854332598,\n",
       "  0.8773150369446311,\n",
       "  0.8739024039153591,\n",
       "  0.8726308718391632,\n",
       "  0.873659461145366,\n",
       "  0.874787073246803,\n",
       "  0.8748830402341594,\n",
       "  0.8730025433082201,\n",
       "  0.8732664715197467,\n",
       "  0.87477806036758],\n",
       " 'mean_accuracy': 0.874374074795429,\n",
       " 'name': 'character 6-grams',\n",
       " 'shape': (416809, 773532)}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_6gram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(6,6))\n",
    "result = get_metric(vectorizer_character_6gram, np.array(trainset.data), np.array(trainset.target), \"character 6-grams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8736925439017369,\n",
       "  0.8703579311006622,\n",
       "  0.8731586776066408,\n",
       "  0.8717671896741999,\n",
       "  0.8700367073726638,\n",
       "  0.8704925505626064,\n",
       "  0.8684292603344449,\n",
       "  0.8723067325687412,\n",
       "  0.8704112481405057,\n",
       "  0.8724746868851673],\n",
       " 'mean_accuracy': 0.871312752814737,\n",
       " 'name': 'character 7-grams',\n",
       " 'shape': (416809, 1710117)}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_7gram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(7,7))\n",
    "\n",
    "result = get_metric(vectorizer_character_7gram, np.array(trainset.data), np.array(trainset.target), \"character 7-grams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracies': [0.8691584300930812,\n",
       "  0.8681268592265617,\n",
       "  0.8658893527181997,\n",
       "  0.8656014586632119,\n",
       "  0.8677574914229506,\n",
       "  0.8659101269163407,\n",
       "  0.866533912334157,\n",
       "  0.8647727818033495,\n",
       "  0.8675080378137147,\n",
       "  0.8680358942367676],\n",
       " 'mean_accuracy': 0.8669294345228336,\n",
       " 'name': 'character 8-grams',\n",
       " 'shape': (416809, 3179100)}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_character_8gram = TfidfVectorizer(stop_words=other_exclusions,\n",
    "                                       analyzer='char',\n",
    "                                       ngram_range=(8,8))\n",
    "result = get_metric(vectorizer_character_8gram, np.array(trainset.data), np.array(trainset.target), \"character 8-grams\")\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metric_oracle(X_raw, y_raw, vectorizers):\n",
    "    results = {\"oracle\":{}}\n",
    "    for train, test in cv.split(X_raw, y_raw):\n",
    "        y_train = y_raw[train]\n",
    "        X_train = X_raw[train]\n",
    "        \n",
    "        y_test = y_raw[test]\n",
    "        X_test = X_raw[test]\n",
    "        \n",
    "        y_pred_oracle = []\n",
    "        for name in vectorizers:\n",
    "            vectorizer = vectorizers[name]\n",
    "            if name in results:\n",
    "                result = results[name]\n",
    "            else:\n",
    "                result = {}\n",
    "                results[name] = result\n",
    "                \n",
    "            X_train_tr = vectorizer.fit_transform(X_train)\n",
    "            \n",
    "            if not \"shape\" in result:\n",
    "                result[\"shape\"] = []\n",
    "            result['shape'].append(X_train_tr.shape)\n",
    "            classifier.fit(X_train_tr, y_train)\n",
    "            X_test_tr = vectorizer.transform(X_test)\n",
    "            y_preds = classifier.predict(X_test_tr)\n",
    "            accuracy = accuracy_score(y_test, y_preds)\n",
    "            \n",
    "            if not \"accuracies\" in result:\n",
    "                result[\"accuracies\"] = []           \n",
    "            \n",
    "            result['accuracies'].append(accuracy)\n",
    "            \n",
    "            if not \"y_preds\" in result:\n",
    "                result[\"y_preds\"] = []\n",
    "                \n",
    "            result['y_preds'].append(y_preds)   \n",
    "            \n",
    "            y_pred_oracle.append(y_preds)\n",
    "            \n",
    "        y_pred_oracle = np.matrix(y_pred_oracle).T\n",
    "        oracle_correct_pred = 0\n",
    "        oracle_incorrect_index = []\n",
    "        for i, yt in enumerate(y_test):\n",
    "            if True in  (y_pred_oracle[i,:] == yt):\n",
    "                 oracle_correct_pred += 1\n",
    "            else:\n",
    "                oracle_incorrect_index.append(test[i])\n",
    "                \n",
    "        accuracy = oracle_correct_pred/len(y_test)\n",
    "        print(\"Oracle classifier accuracy={}\".format(accuracy))\n",
    "        result = results[\"oracle\"]\n",
    "    \n",
    "        if not \"accuracies\" in result:\n",
    "            result[\"accuracies\"] = []           \n",
    "            \n",
    "        result['accuracies'].append(accuracy)\n",
    "        \n",
    "        if not \"oracle_incorrect_index\" in result:\n",
    "            result[\"oracle_incorrect_index\"] = []  \n",
    "            \n",
    "        result['oracle_incorrect_index'] = oracle_incorrect_index\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oracle classifier accuracy=0.9312446022454659\n",
      "Oracle classifier accuracy=0.9294453507340946\n",
      "Oracle classifier accuracy=0.9295859123842426\n",
      "Oracle classifier accuracy=0.9295859123842426\n",
      "Oracle classifier accuracy=0.9305438929008422\n",
      "Oracle classifier accuracy=0.9279287924953816\n",
      "Oracle classifier accuracy=0.9291523715841751\n",
      "Oracle classifier accuracy=0.9278516243581746\n",
      "Oracle classifier accuracy=0.9277316569892989\n",
      "Oracle classifier accuracy=0.9281395460434761\n"
     ]
    }
   ],
   "source": [
    "vectorizers = {\"vectorizer_character_8gram\":vectorizer_character_8gram,\n",
    "              \"vectorizer_character_7gram\":vectorizer_character_7gram,\n",
    "              \"vectorizer_character_6gram\":vectorizer_character_6gram,\n",
    "              \"vectorizer_character_5gram\":vectorizer_character_5gram,\n",
    "              \"vectorizer_character_4gram\":vectorizer_character_4gram,\n",
    "              \"vectorizer_1skipbigram\": vectorizer_1skipbigram,\n",
    "              \"vectorizer_2skipbigram\": vectorizer_2skipbigram,\n",
    "              \"vectorizer_3skipbigram\": vectorizer_3skipbigram,\n",
    "              \"vectorizer_unigrams\": vectorizer_unigrams,\n",
    "              \"vectorizer_bigrams\": vectorizer_bigrams,\n",
    "              \"vectorizer_trigrams\": vectorizer_trigrams}\n",
    "               \n",
    "results = get_metric_oracle(np.array(trainset.data), np.array(trainset.target), vectorizers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2995\n"
     ]
    }
   ],
   "source": [
    "incorrect_indexes = sorted(set(results[\"oracle\"][\"oracle_incorrect_index\"]))\n",
    "print(len(incorrect_indexes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i actually did a good job teaching them and or...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i agreed many months ago and as the time got c...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i also had a gazillion other things that just ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i also kind of stop keeping up with blogs when...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i also often feel a little overwhelmed by my n...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i am also noticing that i can only handle so m...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>i am assuming you guys too feel if you think i...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>i am avoiding spending money it definitely fee...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>i am constantly feeling overwhelmed about my f...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>i am enough even when i feel weird</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>i am extremely happy with is my work in the cr...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>i am feeling a little overwhelmed by the thoug...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>i am feeling kind of overwhelmed by all of thi...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>i am feeling much more then i have in my life ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>i am feeling overwhelmed i dont feel hopeless ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>i am feeling overwhelmed i go back and read th...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>i am feeling overwhelmed i have used a new for...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>i am happy and healthy and i feel amazing and ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>i am happy with and that feels amazing</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>i am just a youngster at the age of on this pl...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>i am not sure if it is the macklemore blaring ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>i am really lack of love that caused me feelin...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>i am seriously loving feeling amazing and i am...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>i am so thrilled because i feel like this will...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>i am still feeling that strange high feeling t...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>i am the number one cheerleader of color and t...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>i am tired or in an unresourceful state i feel...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>i am using git everything feels so strange</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>i approached it i could feel a strange sense o...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>i begged my husband for it last year as if i t...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2965</th>\n",
       "      <td>i wasnt going because i didnt feel like it bec...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2966</th>\n",
       "      <td>i were an overweight teen boy i wouldn t feel ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2967</th>\n",
       "      <td>i will feel a lot less stressed too because he...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2968</th>\n",
       "      <td>i will post fairly often since im going to be ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2969</th>\n",
       "      <td>i will remember this email i will remember it ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2970</th>\n",
       "      <td>i wish that i could just stop time for even ju...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2971</th>\n",
       "      <td>i wish there was more i could do in this situa...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2972</th>\n",
       "      <td>i wish there was something i could do sitting ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2973</th>\n",
       "      <td>i would spend hours and days and weeks and mon...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2974</th>\n",
       "      <td>ill be back to feeling helpless and a little l...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2975</th>\n",
       "      <td>im able to go i feel so stressed and pressured</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2976</th>\n",
       "      <td>im aggravated with myself that i went so long ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2977</th>\n",
       "      <td>im ever feeling stressed i whack heart on blac...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2978</th>\n",
       "      <td>im feeling a bit sad that i havent gotten to s...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2979</th>\n",
       "      <td>im feeling all emotional and shit so i cant be...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2980</th>\n",
       "      <td>im feeling helpless to ever catch up</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2981</th>\n",
       "      <td>im feeling stressed is to ask me to bake a pie...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2982</th>\n",
       "      <td>im feeling stressed out because there arent an...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2983</th>\n",
       "      <td>im feeling stressed with her not answering eit...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2984</th>\n",
       "      <td>im just starting to feel a little physically i...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2985</th>\n",
       "      <td>im not actually feeling too money stressed yet</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2986</th>\n",
       "      <td>im not feeling stressed at all</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2987</th>\n",
       "      <td>im really feeling helpless</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2988</th>\n",
       "      <td>im so tired of feeling stressed and overwhelmed</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989</th>\n",
       "      <td>im starting to feel inhibited</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2990</th>\n",
       "      <td>im starting to feel positively assaulted by th...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2991</th>\n",
       "      <td>ive been feeling a little stressed this week a...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2992</th>\n",
       "      <td>ive been feeling pretty stressed about leaving...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2993</th>\n",
       "      <td>ive been finishing up the final touches on bub...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2994</th>\n",
       "      <td>ive never been capable of feeling that i love ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2995 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0     i actually did a good job teaching them and or...      5\n",
       "1     i agreed many months ago and as the time got c...      5\n",
       "2     i also had a gazillion other things that just ...      5\n",
       "3     i also kind of stop keeping up with blogs when...      5\n",
       "4     i also often feel a little overwhelmed by my n...      5\n",
       "5     i am also noticing that i can only handle so m...      5\n",
       "6     i am assuming you guys too feel if you think i...      5\n",
       "7     i am avoiding spending money it definitely fee...      5\n",
       "8     i am constantly feeling overwhelmed about my f...      5\n",
       "9                    i am enough even when i feel weird      5\n",
       "10    i am extremely happy with is my work in the cr...      5\n",
       "11    i am feeling a little overwhelmed by the thoug...      5\n",
       "12    i am feeling kind of overwhelmed by all of thi...      5\n",
       "13    i am feeling much more then i have in my life ...      5\n",
       "14    i am feeling overwhelmed i dont feel hopeless ...      5\n",
       "15    i am feeling overwhelmed i go back and read th...      5\n",
       "16    i am feeling overwhelmed i have used a new for...      5\n",
       "17    i am happy and healthy and i feel amazing and ...      5\n",
       "18               i am happy with and that feels amazing      5\n",
       "19    i am just a youngster at the age of on this pl...      5\n",
       "20    i am not sure if it is the macklemore blaring ...      5\n",
       "21    i am really lack of love that caused me feelin...      5\n",
       "22    i am seriously loving feeling amazing and i am...      5\n",
       "23    i am so thrilled because i feel like this will...      5\n",
       "24    i am still feeling that strange high feeling t...      5\n",
       "25    i am the number one cheerleader of color and t...      5\n",
       "26    i am tired or in an unresourceful state i feel...      5\n",
       "27           i am using git everything feels so strange      5\n",
       "28    i approached it i could feel a strange sense o...      5\n",
       "29    i begged my husband for it last year as if i t...      5\n",
       "...                                                 ...    ...\n",
       "2965  i wasnt going because i didnt feel like it bec...      4\n",
       "2966  i were an overweight teen boy i wouldn t feel ...      4\n",
       "2967  i will feel a lot less stressed too because he...      4\n",
       "2968  i will post fairly often since im going to be ...      4\n",
       "2969  i will remember this email i will remember it ...      4\n",
       "2970  i wish that i could just stop time for even ju...      4\n",
       "2971  i wish there was more i could do in this situa...      4\n",
       "2972  i wish there was something i could do sitting ...      4\n",
       "2973  i would spend hours and days and weeks and mon...      4\n",
       "2974  ill be back to feeling helpless and a little l...      4\n",
       "2975     im able to go i feel so stressed and pressured      4\n",
       "2976  im aggravated with myself that i went so long ...      4\n",
       "2977  im ever feeling stressed i whack heart on blac...      4\n",
       "2978  im feeling a bit sad that i havent gotten to s...      4\n",
       "2979  im feeling all emotional and shit so i cant be...      4\n",
       "2980               im feeling helpless to ever catch up      4\n",
       "2981  im feeling stressed is to ask me to bake a pie...      4\n",
       "2982  im feeling stressed out because there arent an...      4\n",
       "2983  im feeling stressed with her not answering eit...      4\n",
       "2984  im just starting to feel a little physically i...      4\n",
       "2985     im not actually feeling too money stressed yet      4\n",
       "2986                     im not feeling stressed at all      4\n",
       "2987                         im really feeling helpless      4\n",
       "2988    im so tired of feeling stressed and overwhelmed      4\n",
       "2989                      im starting to feel inhibited      4\n",
       "2990  im starting to feel positively assaulted by th...      4\n",
       "2991  ive been feeling a little stressed this week a...      4\n",
       "2992  ive been feeling pretty stressed about leaving...      4\n",
       "2993  ive been finishing up the final touches on bub...      4\n",
       "2994  ive never been capable of feeling that i love ...      4\n",
       "\n",
       "[2995 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_incorrect = np.array(trainset.data)[incorrect_indexes]\n",
    "y_incorrect = np.array(trainset.target)[incorrect_indexes]\n",
    "incorrect_classified = pd.DataFrame()\n",
    "incorrect_classified[\"text\"] = X_incorrect\n",
    "incorrect_classified[\"label\"] = y_incorrect\n",
    "incorrect_classified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    699\n",
       "3    664\n",
       "1    586\n",
       "5    374\n",
       "0    359\n",
       "4    313\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incorrect_classified.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>0.929121</td>\n",
       "      <td>oracle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.885732</td>\n",
       "      <td>vectorizer_unigrams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>0.882339</td>\n",
       "      <td>vectorizer_character_4gram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>0.877203</td>\n",
       "      <td>vectorizer_character_5gram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>0.874460</td>\n",
       "      <td>vectorizer_character_6gram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11</td>\n",
       "      <td>0.871850</td>\n",
       "      <td>vectorizer_character_7gram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9</td>\n",
       "      <td>0.870603</td>\n",
       "      <td>vectorizer_3skipbigram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4</td>\n",
       "      <td>0.867592</td>\n",
       "      <td>vectorizer_character_8gram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3</td>\n",
       "      <td>0.866658</td>\n",
       "      <td>vectorizer_2skipbigram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5</td>\n",
       "      <td>0.859909</td>\n",
       "      <td>vectorizer_1skipbigram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>0.841433</td>\n",
       "      <td>vectorizer_bigrams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>0.730229</td>\n",
       "      <td>vectorizer_trigrams</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  accuracy                        name\n",
       "0       7  0.929121                      oracle\n",
       "1       2  0.885732         vectorizer_unigrams\n",
       "2      10  0.882339  vectorizer_character_4gram\n",
       "3       8  0.877203  vectorizer_character_5gram\n",
       "4       6  0.874460  vectorizer_character_6gram\n",
       "5      11  0.871850  vectorizer_character_7gram\n",
       "6       9  0.870603      vectorizer_3skipbigram\n",
       "7       4  0.867592  vectorizer_character_8gram\n",
       "8       3  0.866658      vectorizer_2skipbigram\n",
       "9       5  0.859909      vectorizer_1skipbigram\n",
       "10      1  0.841433          vectorizer_bigrams\n",
       "11      0  0.730229         vectorizer_trigrams"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary = []\n",
    "for name in results:\n",
    "    result = results[name]\n",
    "    accuracies = result[\"accuracies\"]\n",
    "    summary.append({\"name\": name, \"accuracy\":np.mean(accuracies)})\n",
    "df_summary = pd.DataFrame(summary)\n",
    "df_summary = df_summary.sort_values(by=['accuracy'],ascending=False)\n",
    "df_summary = df_summary.reset_index()\n",
    "df_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
